import re
import xlwt
import time
import requests
from PIL import Image 
from bs4 import BeautifulSoup
#from lxml import etree

item_value = []
count = 1
browser = requests.session()
requestVerifyCodeUrl = "https://mall.jd.com/sys/vc/createVerifyCode.html"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'}

def parserUrl(url, params={}):
    #解析URL
    global headers
    req = requests.get(url, headers=headers, params=params)
    #time.sleep(1)
    soup = BeautifulSoup(req.text, "lxml")
    return soup

def enterprisePage(url):
    #处理企业详情页面相关数据
    while True:
        global headers
        global requestVerifyCodeUrl
        req = browser.get(requestVerifyCodeUrl, headers=headers)
        #保存验证码图片
        f = open("VerifyCodeImage.jpg", 'wb')
        f.write(req.content)
        f.close()
        f1 = open("VerifyCodeImage.jpg", 'rb')
        image = Image.open(f1)
        image.show()
        f1.close()
        #手动输入
        vc = input('请输入验证码:')
        datas = {'verifyCode' : vc}
        req2 = browser.post(url, data=datas, headers=headers)
        soup = BeautifulSoup(req2.text, "lxml")
        #判断若是有form的代码块则证明没有验证成功
        if soup.find('form'):
            print('验证码输入错误！请重新输入！')
            continue
        infoList = soup.find_all('li', attrs={'class':'noBorder'})#获取企业相关信息代码段
        for info in infoList:
            if info.find('label', attrs={'class':'jLabel'}):
                if info.find('label', attrs={'class':'jLabel'}).text == "企业名称： ":
                    item_value.append(info.find('span').text)#企业名称
                if info.find('label', attrs={'class':'jLabel'}).text == "公司地址：":
                    item_value.append(info.find('span').text)#企业地址
        return

def itemPage(url):
    #处理商品详情页面相关数据
    soup = parserUrl(url)
    clearfixList = soup.find_all('dl', attrs={'class':'clearfix'})#获取重量相关代码段
    for clearfix in clearfixList:
        if clearfix.find('dt').text == "净含量":
            weight = re.sub("[a-zA-Z]", "", clearfix.find('dd').text)
            return weight
    return None

def storePage(url):
    #处理店铺详情页面相关数据
    storID = re.sub("[^0-9]", "", url)
    enterpriseUrl = 'https://mall.jd.com/showLicence-' + str(storID) + '.html'
    enterprisePage(enterpriseUrl)

def mainPage(sheet, n):
    #处理搜索页面相关数据
    url = "https://search.jd.com/search?"
    params = {"keyword":"8424西瓜", 'page':str(n), 'scrolling':'y'}
    soup = parserUrl(url, params)
    item_list = soup.find_all("li", attrs={"class":"gl-item"})
    for item in item_list:
        p_name = item.find('div', attrs={"class":"p-name p-name-type-2"})#获取商品相关代码段
        itemName = p_name.find('em').text
        itemUrl = 'https:' + p_name.find('a')['href']
        p_price = item.find('div', attrs={"class":"p-price"})#获取价格相关代码段
        itemPrice = p_price.find('i').text
        p_shop = item.find('div', attrs={"class":"p-shop"})#获取店铺相关代码段
        storeName = p_shop.find('a').text
        storeUrl = 'https:' + p_shop.find('a')['href']
        
        item_value.clear()
        item_value.append(storeName)#店铺名字
        item_value.append(storeUrl)#店铺URL
        item_value.append(itemName)#商品名字
        item_value.append(itemUrl)#商品URl
        storePage(storeUrl)#在店铺页面处理数据
        item_value.append(itemPrice)#商品价格
        itemWeight = itemPage(itemUrl)#在商品页面处理数据
        if itemWeight:
            item_value.append(itemWeight)#商品重量
        else :
            item_value.append(0)
        if itemWeight:
            itemUnitPrice = float(itemPrice) / float(itemWeight) / 2
            item_value.append('%.2f' %itemUnitPrice)#商品单价
        else :
            item_value.append(0)
            
        #写入excel表
        global count
        for i in range(0, len(item_value)):
            sheet.write(count, i, item_value[i])
        count += 1
        if count == 4: return
    if n%2 == 0:
        print("******已完成第" + str(n/2) +"页商品信息写入******")

def main():
    n = input("请输入要爬取的页数：")
    n = int(n) * 2
    print("**************正在写入**************")
    print("请稍候...")
    value_title = ['店铺名称','店铺URL','商品名称','商品URL','企业名称','企业地址','价格(元)','重量(kg)','单价(元/500g)']
    #value_title = ['店铺名称','店铺URL','商品名称','商品URL','价格(元)','重量(kg)','单价(元/500g)']
    workbook = xlwt.Workbook()
    sheet = workbook.add_sheet('8424西瓜')
    for i in range(0, len(value_title)):
        sheet.write(0, i, value_title[i])#写入表头
    for i in range(1, n+1):
        mainPage(sheet, i)
        break
    workbook.save('京东商品数据.xls')
    print("**************写入完成**************")

if __name__ == '__main__':
    main()
